package com.wish.utils;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.atomic.AtomicInteger;

/***
 * 
 * title: 高效亿级txt文件 切分
 *
 * @author HadLuo
 * @date 2021-5-19 11:47:37
 */
public class SplitBigFileTools {

    public static void main(String[] args) throws CloneNotSupportedException, InterruptedException, IOException {
        // 读者测试请更换 磁盘地址！！！！！！！！！！！！！！！
        int segment = 20;
        // 将文件拆分
        long s = System.currentTimeMillis();
        splitFile("F:\\last.txt", 20);
        System.out.println("拆分文件耗時： " + (System.currentTimeMillis() - s));
        AtomicInteger incr = new AtomicInteger(1);
        ExecutorService service = Executors.newFixedThreadPool(segment);

        AtomicInteger total = new AtomicInteger(0);
        // 多线程 分批读取文件
        CountDownLatch downLatch = new CountDownLatch(segment);
        for (int i = 1; i <= segment; i++) {
            service.execute(() -> {
                try {
                    readFile("F:\\last.txt" + incr.getAndIncrement(), line -> {
                        // 这里千万不要 打印 line ， 太耗时了
                        total.getAndIncrement();
                    });
                } finally {
                    downLatch.countDown();
                }
            });
        }
        downLatch.await();
        System.out.println("读取文件总耗时：" + (System.currentTimeMillis() - s) + " ,读取到的总行数：" + total.get());
        service.shutdown();
    }

    public static void splitFile(String filePath, int fileCount) throws IOException {
        FileInputStream fis = new FileInputStream(filePath);
        FileChannel inputChannel = fis.getChannel();
        final long fileSize = inputChannel.size();
        long average = fileSize / fileCount;// 平均值
        long bufferSize = 50 * 1024 * 1024; // 缓存块大小，自行调整
        ByteBuffer byteBuffer = ByteBuffer.allocate(Integer.valueOf(bufferSize + "")); // 申请一个缓存区
        long startPosition = 0; // 子文件开始位置
        long endPosition = average < bufferSize ? 0 : average - bufferSize;// 子文件结束位置
        for (int i = 0; i < fileCount; i++) {
            if (i + 1 != fileCount) {
                int read = inputChannel.read(byteBuffer, endPosition);// 读取数据
                readW:
                while (read != -1) {
                    byteBuffer.flip();// 切换读模式
                    byte[] array = byteBuffer.array();
                    for (int j = 0; j < array.length; j++) {
                        byte b = array[j];
                        if (b == 10 || b == 13) { // 判断\n\r
                            endPosition += j;
                            break readW;
                        }
                    }
                    endPosition += bufferSize;
                    byteBuffer.clear(); // 重置缓存块指针
                    read = inputChannel.read(byteBuffer, endPosition);
                }
            } else {
                endPosition = fileSize; // 最后一个文件直接指向文件末尾
            }
            FileOutputStream fos = new FileOutputStream(filePath + (i + 1));
            FileChannel outputChannel = fos.getChannel();
            inputChannel.transferTo(startPosition, endPosition - startPosition, outputChannel);// 通道传输文件数据
            outputChannel.close();
            fos.close();
            startPosition = endPosition + 1;
            endPosition += average;
        }
        inputChannel.close();
        fis.close();

    }

    public interface Callback {
        public void onReceive(String line);
    }

    /**
     * 大文件读取
     * 
     * @param filePath
     */
    public static void readFile(String filePath, Callback callback) {
        File file = new File(filePath);
        BufferedReader reader = null;
        try {
            reader = new BufferedReader(new FileReader(file), 20 * 1024 * 1024); // 如果是读大文件，设置缓存
            String tempString = null;
            while ((tempString = reader.readLine()) != null) {
                callback.onReceive(tempString);
            }
            reader.close();
        } catch (IOException e) {
            e.printStackTrace();
        } finally {
            if (reader != null) {
                try {
                    reader.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
        }
    }

}
